In [1]:
import numpy as np
from sklearn.datasets import make_regression
from scipy.spatial.distance import norm
from itertools import product
from collections import OrderedDict
from plotly.graph_objs import *
import plotly.tools as tls
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode()
import time

from plot_helpers import *

X, y = make_regression(n_samples = int(1e5), n_features = 2, n_informative=2, random_state=0, noise=10)
X = (X - X.mean(axis=0))/X.std()
In [2]:
# LINEAR REGRESSION WITH L2 REGULARIZATION

LAMBDA_= 10000.

def ridge_cost_function(X, y, params, lambda_=LAMBDA_):
    '''
    OLS from linear regression
    '''
    n_observations = X.shape[0]
    avg_squared_residuals = (((predict(X, params) - y)**2).sum()
                             + lambda_*(params**2).sum())/(2*n_observations)
    return avg_squared_residuals

def ridge_gradient_of_cost_function(X, y, params, lambda_=LAMBDA_):
    n_observations = X.shape[0]
    gradient = ((predict(X, params) - y).dot(X)
               + lambda_*params)/n_observations
    return gradient

ridge_param_history, ridge_time_history = gradient_descent(X, y, ridge_cost_function, ridge_gradient_of_cost_function,
                                   initial_guess = np.array([0., 0.]))

figure_3d = plot_results(X, y, ridge_cost_function, ridge_param_history)
iplot(figure_3d)
Final gradient of cost function [-0.0008337  -0.00045728]
Final params [ 89.18209675  50.4859195 ]